package top.rushpeak.edu03.crawler.lottery;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

public class LotteryController {

	public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
	
    public static void main(String[] args) throws Exception {
        String crawlStorageFolder = "/Users/wesley/output/crawl";
        int numberOfCrawlers = 1;

        CrawlConfig config = new CrawlConfig();
        config.setCrawlStorageFolder(crawlStorageFolder);
        config.setIncludeBinaryContentInCrawling(false);//开关二进制文件爬取
        config.setMaxDepthOfCrawling(1);
        config.setMaxPagesToFetch(1);
        config.setPolitenessDelay(2000);
        config.setUserAgentString(USER_AGENT);//伪装浏览器UserAgent
        /*
         * 初始化爬虫控制器
         * Instantiate the controller for this crawl.
         */
        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        robotstxtConfig.setUserAgentName(USER_AGENT);
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

        /*
         * 为每个爬虫添加种子域名
         * 这些是所要爬取的一级页面，爬虫会通过这些页面上的链接继续向下爬取
         * For each crawl, you need to add some seed urls. These are the first
         * URLs that are fetched and then the crawler starts following links
         * which are found in these pages
         */
    	controller.addSeed(LotteryCrawler.TOURL);
    	
        /*
         * 启动爬取。
         * Start the crawl. This is a blocking operation, meaning that your code
         * will reach the line after this only when crawling is finished.
         */
        controller.start(LotteryCrawler.class, numberOfCrawlers);
        
        controller.shutdown();
    }
}
